import csv, os, math
import numpy



################################################
##JOHNTALBOT
##THIS SCRIPT CLEANS DATA 


    
################################################

def main():

    global total_trans_list
    global total_rate_list
    
    global early_trans_list
    global early_rate_list
    
    global later_trans_list
    global later_rate_list
    global path
    total_trans_list=[]
    total_rate_list=[]
    
    early_trans_list=[]
    early_rate_list=[]
    
    later_trans_list=[]
    later_rate_list=[]
    path = "E:\\cp\\csv\\"
    print""
    print"Okay"
    print""
    
    writer= open("E:\\cp\\csv\\testStd.csv", 'w')
    for i in range(13,15):
        print i
        writer.write("ISLAND "+str(i)+"\n\n"+"Transect,1850+,1925+,1970+\n")
        total(i)
        early(i)
        later(i)
        totalc=0
        for z in total_trans_list:
            writer.write(str(z)+","+str(str(total_rate_list[totalc])+","))

            earlyc=0
            switch=0
            for y in early_trans_list:
                if str(y)==str(z):
                    writer.write(str(early_rate_list[earlyc])+",")
                    switch=1
                earlyc=earlyc+1
            if switch==0:
                writer.write("NULL,")
                
            laterc=0
            switch=0
            for q in later_trans_list:
                if str(q)==str(z):
                    writer.write(str(later_rate_list[laterc])+"\n")
                    switch=1
                laterc=laterc+1
            if switch==0:
                writer.write("NULL\n")
            
            
            totalc=totalc+1
        writer.write("\n\n\n")
            
        
        del total_trans_list[:]
        del total_rate_list[:]
    
        del early_trans_list[:]
        del early_rate_list[:]
    
        del later_trans_list[:]
        del later_rate_list[:]
    writer.close()
   
    openIt()
   

    

    
  
def total(num):
    total_trans_listr=[]
    total_rate_listr=[]
    
    islandTotal='r'+str(num)+".csv"
    with open(path+islandTotal,'rb') as csvTotal:
        total = csv.reader(csvTotal)
        for x in total:
                        
                
            if (str(x[8]) != "-1.#IND") & (str(x[8]) != "LRR"):
                total_trans_list.append(x[1])
                total_rate_list.append(float(x[8]))

        stddev=numpy.std(total_rate_list)
        mean=numpy.mean(total_rate_list)
     
        max=(stddev*2)+mean
        min=mean-(stddev*2)
        count=0
        print max, min
        for fix in total_rate_list:
            if (float(fix )> max) | (float(fix) < min):
                print "removing" ,count, fix
                total_rate_listr.append(total_rate_list[count])
                total_trans_listr.append(total_trans_list[count])
                
            count=count+1
        for remove in total_rate_listr:
            total_rate_list.remove(remove)
        for remove in total_trans_listr:
            total_trans_list.remove(remove)
                
   
    
 
def early(num):

    early_trans_listr=[]
    early_rate_listr=[]

    island='Rates'+str(num)+".csv"
    with open(path+island,'rb') as csvTotal:
        total = csv.reader(csvTotal)
        for x in total:
                        
                
            if (str(x[8]) != "-1.#IND") & (str(x[8]) != "LRR"):
                early_trans_list.append(x[1])
                early_rate_list.append(float(x[8]))

     
        stddev=numpy.std(early_rate_list)
        mean=numpy.mean(early_rate_list)
        
        max=(stddev*2)+mean
        min=mean-(stddev*2)
        print max, min
        count=0
        for fix in early_rate_list:
            if (float(fix) > float(max)) | (float(fix) < float(min)):
                print "removing" ,count, fix
                early_rate_listr.append(early_rate_list[count])
                early_trans_listr.append(early_trans_list[count])
                
            count=count+1
        
        for remove in early_rate_listr:
            early_rate_list.remove(remove)
        for remove in early_trans_listr:
            early_trans_list.remove(remove)
            
            
                
  

def later(num):

    later_trans_listr=[]
    later_rate_listr=[]

    island='later'+str(num)+".csv"
    with open(path+island,'rb') as csvTotal:
        total = csv.reader(csvTotal)
        for x in total:
                   
                
            if (str(x[8]) != "-1.#IND") & (str(x[8]) != "LRR"):
                later_trans_list.append(x[1])
                later_rate_list.append(float(x[8]))

                
        stddev=numpy.std(later_rate_list)
        mean=numpy.mean(later_rate_list)
      
        max=(stddev*2)+mean
        min=mean-(stddev*2)
        print max, min
        count=0
        for fix in later_rate_list:
            if (float(fix) > max) | (float(fix) < min):
                print "removing" ,count, fix
                later_rate_listr.append(later_rate_list[count])
                later_trans_listr.append(later_trans_list[count])
                
            count=count+1
        for remove in later_rate_listr:
            later_rate_list.remove(remove)
        for remove in later_trans_listr:
            later_trans_list.remove(remove)
    
   
        
            
def openIt():
    os.startfile("E:\\cp\\csv\\testStd.csv")


          
main()
